Problem 6.3¶
Importing packages¶
In [ ]:
import numpy as np
from sklearn.svm import SVR
from sklearn import svm
from sklearn.metrics.pairwise import linear_kernel, polynomial_kernel, rbf_kernel
from sklearn.metrics import mean_squared_error
import matplotlib.pyplot as plt
1) Construct a train dataset and represent them.¶
In [ ]:
def data(N,a):
N=N+3
h=np.array([1,a])
y=np.sign(np.random.randn(N,1))
y=np.reshape(y,len(y))
t=np.arange(N)
z=np.convolve(h,y)
z=z[1:N-1]
y=y[2:N-1]
X=np.array([z[0:N-3],z[1:N-2]])+0.2*np.random.randn(2,N-3)
return X,y
data_alpha = 1.5 # Parameter 'a' for data generation
N_train = 100 # Number of training samples
X_train, y_train = data(N_train, data_alpha)
N_Val = 110 # Number of validation samples
X_Val, y_Val = data(N_Val, data_alpha)
N_test = 1000 # Number of test samples
X_test, y_test = data(N_test, data_alpha)
print("Shape of X_train: ", X_train.shape)
print("Shape of y_train: ", y_train.shape)
print("Shape of X_Val: ", X_Val.shape)
print("Shape of y_Val: ", y_Val.shape)
print("Shape of X_test: ", X_test.shape)
print("Shape of y_test: ", y_test.shape)
plt.figure(figsize=(15, 5))
plt.subplot(131)
ind1=np.where(y_train==1)
ind2=np.where(y_train==-1)
plt.plot(X_train[0,ind1],X_train[1,ind1],'b*')
plt.plot(X_train[0,ind2],X_train[1,ind2],'r*')
plt.xlabel('x[n-1]')
plt.ylabel('x[n]')
plt.title('Training Data')
plt.subplot(132)
ind1=np.where(y_test==1)
ind2=np.where(y_test==-1)
plt.plot(X_test[0,ind1],X_test[1,ind1],'b*')
plt.plot(X_test[0,ind2],X_test[1,ind2],'r*')
plt.xlabel('x[n-1]')
plt.ylabel('x[n]')
plt.title('Test Data')
plt.subplot(133)
ind1=np.where(y_Val==1)
ind2=np.where(y_Val==-1)
plt.plot(X_Val[0,ind1],X_Val[1,ind1],'b*')
plt.plot(X_Val[0,ind2],X_Val[1,ind2],'r*')
plt.xlabel('x[n-1]')
plt.ylabel('x[n]')
plt.title('Validation Data')
plt.tight_layout()
plt.show()
Shape of X_train: (2, 100) Shape of y_train: (100,) Shape of X_Val: (2, 110) Shape of y_Val: (110,) Shape of X_test: (2, 1000) Shape of y_test: (1000,)
Linear Kernel¶
In [ ]:
# Computing kernel matrices
K_linear = linear_kernel(X_train.T, X_train.T)
C_high = 5; C_low = -5
C_Values = np.logspace(C_low, C_high, C_high - C_low + 1, base=10)
MMSE_linear = np.zeros(len(C_Values))
total_iterations = len(C_Values)
# Train SVM with precomputed kernel matrices
for i, C in enumerate(C_Values):
clf_linear = svm.SVC(kernel='precomputed', C=C)
# Fit the model
clf_linear.fit(K_linear, y_train)
# Validation set
K_val_linear = linear_kernel(X_Val.T, X_train.T)
# Predict on validation set
y_pred_linear = clf_linear.predict(K_val_linear)
# Compute validation error
val_error_linear = np.mean(y_pred_linear != y_Val)
# MMSE_linear[i] = mean_squared_error(y_Val, y_pred_linear)
MMSE_linear[i] = np.mean((y_pred_linear - y_Val)**2)
# % Complete status update
print("Completed ", i+1, " out of ", total_iterations, " iterations")
# Find the indices of the best hyperparameters
best_index_linear = np.argmin(MMSE_linear)
best_C_linear = C_Values[best_index_linear]
# Training the SVM with best hyperparameters on the test set
clf_linear_best = svm.SVC(kernel='precomputed', C=best_C_linear)
clf_linear_best.fit(K_linear, y_train)
K_test_linear = linear_kernel(X_test.T, X_train.T)
y_pred_linear = clf_linear_best.predict(K_test_linear)
# test_MMSE_linear = mean_squared_error(y_test, y_pred_linear)
test_MMSE_linear = np.mean((y_test - y_pred_linear)**2)
# Print validation errors
print("Best C for Linear Kernel: ", best_C_linear)
print("Validation error for Linear Kernel: ", MMSE_linear[best_index_linear])
# Printing and plotting the results
# Generate a grid of points
x_min_plot = X_test[0, :].min() - 1
x_max_plot = X_test[0, :].max() + 1
y_min_plot = X_test[1, :].min() - 1
y_max_plot = X_test[1, :].max() + 1
x_meshgrid, y_meshgrid = np.meshgrid(np.linspace(x_min_plot, x_max_plot, 100), np.linspace(y_min_plot, y_max_plot, 100))
# Combine x_meshgrid and y_meshgrid into a single matrix
grid_points = np.c_[x_meshgrid.ravel(), y_meshgrid.ravel()]
# Compute kernel values between grid points and training data for each kernel
K_test_linear = linear_kernel(grid_points, X_train.T)
# Predict on grid points for each kernel
Z_linear = clf_linear_best.decision_function(K_test_linear)
# Reshape the decision function values
Z_linear = Z_linear.reshape(x_meshgrid.shape)
# Plot decision boundary for each kernel
plt.figure(figsize=(5, 5))
plt.scatter(X_test[0, y_test == 1], X_test[1, y_test == 1], c='blue', marker='*', label='Class 1')
plt.scatter(X_test[0, y_test == -1], X_test[1, y_test == -1], c='red', marker='*', label='Class -1')
plt.contour(x_meshgrid, y_meshgrid, Z_linear, levels=[0], linewidths=2, colors='blue')
plt.xlabel('x[n-1]')
plt.ylabel('x[n]')
plt.title('Linear Kernel')
plt.show()
Completed 1 out of 11 iterations Completed 2 out of 11 iterations Completed 3 out of 11 iterations Completed 4 out of 11 iterations Completed 5 out of 11 iterations Completed 6 out of 11 iterations Completed 7 out of 11 iterations Completed 8 out of 11 iterations Completed 9 out of 11 iterations Completed 10 out of 11 iterations Completed 11 out of 11 iterations Best C for Linear Kernel: 0.1 Validation error for Linear Kernel: 1.309090909090909
Polynomial Order 3 Kernel¶
In [ ]:
# Kernel Parameter
degree = 3
C_low = -5 ; C_high = 5
C_Values = np.logspace(C_low, C_high, C_high - C_low + 1, base=10)
MMSE_poly = np.zeros(len(C_Values))
# Computing kernel matrices
K_poly = polynomial_kernel(X_train.T, X_train.T, degree=degree)
# Train SVM with precomputed kernel matrices
for i, C in enumerate(C_Values):
clf_poly = svm.SVC(kernel='precomputed', C=C)
# Fit the model
clf_poly.fit(K_poly, y_train)
# Validation set
K_val_poly = polynomial_kernel(X_Val.T, X_train.T, degree=degree)
# Predict on validation set
y_pred_poly = clf_poly.predict(K_val_poly)
# Compute validation error
# MMSE_poly[i] = mean_squared_error(y_Val, y_pred_poly)
MMSE_poly[i] = np.mean((y_pred_poly - y_Val)**2)
# % Complete status update
print("Completed ", i+1, " out of ", total_iterations, " iterations")
# Find the indices of the best hyperparameters
best_index_poly = np.argmin(MMSE_poly)
best_C_poly = C_Values[best_index_poly]
# Training the SVM with best hyperparameters on the test set
clf_poly_best = svm.SVC(kernel='precomputed', C=best_C_poly)
clf_poly_best.fit(K_poly, y_train)
K_test_poly = polynomial_kernel(X_test.T, X_train.T, degree=degree)
y_pred_poly = clf_poly_best.predict(K_test_poly)
# test_MMSE_poly = mean_squared_error(y_test, y_pred_poly)
test_MMSE_poly = np.mean((y_test-y_pred_poly)**2)
# Print Validation errors
print("Best C for Polynomial Kernel: ", best_C_poly)
print("Validation error for Polynomial Kernel: ", test_MMSE_poly)
# Plotting the results
# Generate a grid of points
x_min_plot = X_test[0, :].min() - 1
x_max_plot = X_test[0, :].max() + 1
y_min_plot = X_test[1, :].min() - 1
y_max_plot = X_test[1, :].max() + 1
x_meshgrid, y_meshgrid = np.meshgrid(np.linspace(x_min_plot, x_max_plot, 100), np.linspace(y_min_plot, y_max_plot, 100))
# Combine x_meshgrid and y_meshgrid into a single matrix
grid_points = np.c_[x_meshgrid.ravel(), y_meshgrid.ravel()]
# Compute kernel values between grid points and training data for each kernel
K_test_poly = polynomial_kernel(grid_points, X_train.T, degree=degree)
# Predict on grid points for each kernel
Z_poly = clf_poly_best.decision_function(K_test_poly)
# Reshape the decision function values
Z_poly = Z_poly.reshape(x_meshgrid.shape)
# Plot decision boundary for each kernel
plt.figure(figsize=(5, 5))
plt.scatter(X_test[0, y_test == 1], X_test[1, y_test == 1], c='blue', marker='*', label='Class 1')
plt.scatter(X_test[0, y_test == -1], X_test[1, y_test == -1], c='red', marker='*', label='Class -1')
plt.contour(x_meshgrid, y_meshgrid, Z_poly, levels=[0], linewidths=2, colors='green')
plt.xlabel('x[n-1]')
plt.ylabel('x[n]')
plt.title('Polynomial Kernel (Order 3)')
plt.show()
Completed 1 out of 11 iterations Completed 2 out of 11 iterations Completed 3 out of 11 iterations Completed 4 out of 11 iterations Completed 5 out of 11 iterations Completed 6 out of 11 iterations Completed 7 out of 11 iterations Completed 8 out of 11 iterations Completed 9 out of 11 iterations Completed 10 out of 11 iterations Completed 11 out of 11 iterations Best C for Polynomial Kernel: 0.1 Validation error for Polynomial Kernel: 0.048
In [ ]:
# Kernel Parameter
degree = 3
C_low = 0.1 ; C_high = 2
C_Values = np.linspace(C_low, C_high, num=500)
MMSE_poly = np.zeros(len(C_Values))
# Computing kernel matrices
K_poly = polynomial_kernel(X_train.T, X_train.T, degree=degree)
total_iterations = len(C_Values)
# Train SVM with precomputed kernel matrices
for i, C in enumerate(C_Values):
clf_poly = svm.SVC(kernel='precomputed', C=C)
# Fit the model
clf_poly.fit(K_poly, y_train)
# Validation set
K_val_poly = polynomial_kernel(X_Val.T, X_train.T, degree=degree)
# Predict on validation set
y_pred_poly = clf_poly.predict(K_val_poly)
# Compute validation error
# MMSE_poly[i] = mean_squared_error(y_Val, y_pred_poly)
MMSE_poly[i] = np.mean((y_pred_poly - y_Val)**2)
# % Complete status update
print("Completed ", i+1, " out of ", total_iterations, " iterations")
# Find the indices of the best hyperparameters
best_index_poly = np.argmin(MMSE_poly)
best_C_poly = C_Values[best_index_poly]
# Training the SVM with best hyperparameters on the test set
clf_poly_best = svm.SVC(kernel='precomputed', C=best_C_poly)
clf_poly_best.fit(K_poly, y_train)
K_test_poly = polynomial_kernel(X_test.T, X_train.T, degree=degree)
y_pred_poly = clf_poly_best.predict(K_test_poly)
# test_MMSE_poly = mean_squared_error(y_test, y_pred_poly)
test_MMSE_poly = np.mean((y_test-y_pred_poly)**2)
# Print Validation errors
print("Best C for Polynomial Kernel: ", best_C_poly)
print("Validation error for Polynomial Kernel: ", test_MMSE_poly)
# Plotting the results
# Generate a grid of points
x_min_plot = X_test[0, :].min() - 1
x_max_plot = X_test[0, :].max() + 1
y_min_plot = X_test[1, :].min() - 1
y_max_plot = X_test[1, :].max() + 1
x_meshgrid, y_meshgrid = np.meshgrid(np.linspace(x_min_plot, x_max_plot, 100), np.linspace(y_min_plot, y_max_plot, 100))
# Combine x_meshgrid and y_meshgrid into a single matrix
grid_points = np.c_[x_meshgrid.ravel(), y_meshgrid.ravel()]
# Compute kernel values between grid points and training data for each kernel
K_test_poly = polynomial_kernel(grid_points, X_train.T, degree=degree)
# Predict on grid points for each kernel
Z_poly = clf_poly_best.decision_function(K_test_poly)
# Reshape the decision function values
Z_poly = Z_poly.reshape(x_meshgrid.shape)
# Plot decision boundary for each kernel
plt.figure(figsize=(5, 5))
plt.scatter(X_test[0, y_test == 1], X_test[1, y_test == 1], c='blue', marker='*', label='Class 1')
plt.scatter(X_test[0, y_test == -1], X_test[1, y_test == -1], c='red', marker='*', label='Class -1')
plt.contour(x_meshgrid, y_meshgrid, Z_poly, levels=[0], linewidths=2, colors='green')
plt.xlabel('x[n-1]')
plt.ylabel('x[n]')
plt.title('Polynomial Kernel (Order 3)')
plt.show()
Completed 1 out of 500 iterations Completed 2 out of 500 iterations Completed 3 out of 500 iterations Completed 4 out of 500 iterations Completed 5 out of 500 iterations Completed 6 out of 500 iterations Completed 7 out of 500 iterations Completed 8 out of 500 iterations Completed 9 out of 500 iterations Completed 10 out of 500 iterations Completed 11 out of 500 iterations Completed 12 out of 500 iterations Completed 13 out of 500 iterations Completed 14 out of 500 iterations Completed 15 out of 500 iterations Completed 16 out of 500 iterations Completed 17 out of 500 iterations Completed 18 out of 500 iterations Completed 19 out of 500 iterations Completed 20 out of 500 iterations Completed 21 out of 500 iterations Completed 22 out of 500 iterations Completed 23 out of 500 iterations Completed 24 out of 500 iterations Completed 25 out of 500 iterations Completed 26 out of 500 iterations Completed 27 out of 500 iterations Completed 28 out of 500 iterations Completed 29 out of 500 iterations Completed 30 out of 500 iterations Completed 31 out of 500 iterations Completed 32 out of 500 iterations Completed 33 out of 500 iterations Completed 34 out of 500 iterations Completed 35 out of 500 iterations Completed 36 out of 500 iterations Completed 37 out of 500 iterations Completed 38 out of 500 iterations Completed 39 out of 500 iterations Completed 40 out of 500 iterations Completed 41 out of 500 iterations Completed 42 out of 500 iterations Completed 43 out of 500 iterations Completed 44 out of 500 iterations Completed 45 out of 500 iterations Completed 46 out of 500 iterations Completed 47 out of 500 iterations Completed 48 out of 500 iterations Completed 49 out of 500 iterations Completed 50 out of 500 iterations Completed 51 out of 500 iterations Completed 52 out of 500 iterations Completed 53 out of 500 iterations Completed 54 out of 500 iterations Completed 55 out of 500 iterations Completed 56 out of 500 iterations Completed 57 out of 500 iterations Completed 58 out of 500 iterations Completed 59 out of 500 iterations Completed 60 out of 500 iterations Completed 61 out of 500 iterations Completed 62 out of 500 iterations Completed 63 out of 500 iterations Completed 64 out of 500 iterations Completed 65 out of 500 iterations Completed 66 out of 500 iterations Completed 67 out of 500 iterations Completed 68 out of 500 iterations Completed 69 out of 500 iterations Completed 70 out of 500 iterations Completed 71 out of 500 iterations Completed 72 out of 500 iterations Completed 73 out of 500 iterations Completed 74 out of 500 iterations Completed 75 out of 500 iterations Completed 76 out of 500 iterations Completed 77 out of 500 iterations Completed 78 out of 500 iterations Completed 79 out of 500 iterations Completed 80 out of 500 iterations Completed 81 out of 500 iterations Completed 82 out of 500 iterations Completed 83 out of 500 iterations Completed 84 out of 500 iterations Completed 85 out of 500 iterations Completed 86 out of 500 iterations Completed 87 out of 500 iterations Completed 88 out of 500 iterations Completed 89 out of 500 iterations Completed 90 out of 500 iterations Completed 91 out of 500 iterations Completed 92 out of 500 iterations Completed 93 out of 500 iterations Completed 94 out of 500 iterations Completed 95 out of 500 iterations Completed 96 out of 500 iterations Completed 97 out of 500 iterations Completed 98 out of 500 iterations Completed 99 out of 500 iterations Completed 100 out of 500 iterations Completed 101 out of 500 iterations Completed 102 out of 500 iterations Completed 103 out of 500 iterations Completed 104 out of 500 iterations Completed 105 out of 500 iterations Completed 106 out of 500 iterations Completed 107 out of 500 iterations Completed 108 out of 500 iterations Completed 109 out of 500 iterations Completed 110 out of 500 iterations Completed 111 out of 500 iterations Completed 112 out of 500 iterations Completed 113 out of 500 iterations Completed 114 out of 500 iterations Completed 115 out of 500 iterations Completed 116 out of 500 iterations Completed 117 out of 500 iterations Completed 118 out of 500 iterations Completed 119 out of 500 iterations Completed 120 out of 500 iterations Completed 121 out of 500 iterations Completed 122 out of 500 iterations Completed 123 out of 500 iterations Completed 124 out of 500 iterations Completed 125 out of 500 iterations Completed 126 out of 500 iterations Completed 127 out of 500 iterations Completed 128 out of 500 iterations Completed 129 out of 500 iterations Completed 130 out of 500 iterations Completed 131 out of 500 iterations Completed 132 out of 500 iterations Completed 133 out of 500 iterations Completed 134 out of 500 iterations Completed 135 out of 500 iterations Completed 136 out of 500 iterations Completed 137 out of 500 iterations Completed 138 out of 500 iterations Completed 139 out of 500 iterations Completed 140 out of 500 iterations Completed 141 out of 500 iterations Completed 142 out of 500 iterations Completed 143 out of 500 iterations Completed 144 out of 500 iterations Completed 145 out of 500 iterations Completed 146 out of 500 iterations Completed 147 out of 500 iterations Completed 148 out of 500 iterations Completed 149 out of 500 iterations Completed 150 out of 500 iterations Completed 151 out of 500 iterations Completed 152 out of 500 iterations Completed 153 out of 500 iterations Completed 154 out of 500 iterations Completed 155 out of 500 iterations Completed 156 out of 500 iterations Completed 157 out of 500 iterations Completed 158 out of 500 iterations Completed 159 out of 500 iterations Completed 160 out of 500 iterations Completed 161 out of 500 iterations Completed 162 out of 500 iterations Completed 163 out of 500 iterations Completed 164 out of 500 iterations Completed 165 out of 500 iterations Completed 166 out of 500 iterations Completed 167 out of 500 iterations Completed 168 out of 500 iterations Completed 169 out of 500 iterations Completed 170 out of 500 iterations Completed 171 out of 500 iterations Completed 172 out of 500 iterations Completed 173 out of 500 iterations Completed 174 out of 500 iterations Completed 175 out of 500 iterations Completed 176 out of 500 iterations Completed 177 out of 500 iterations Completed 178 out of 500 iterations Completed 179 out of 500 iterations Completed 180 out of 500 iterations Completed 181 out of 500 iterations Completed 182 out of 500 iterations Completed 183 out of 500 iterations Completed 184 out of 500 iterations Completed 185 out of 500 iterations Completed 186 out of 500 iterations Completed 187 out of 500 iterations Completed 188 out of 500 iterations Completed 189 out of 500 iterations Completed 190 out of 500 iterations Completed 191 out of 500 iterations Completed 192 out of 500 iterations Completed 193 out of 500 iterations Completed 194 out of 500 iterations Completed 195 out of 500 iterations Completed 196 out of 500 iterations Completed 197 out of 500 iterations Completed 198 out of 500 iterations Completed 199 out of 500 iterations Completed 200 out of 500 iterations Completed 201 out of 500 iterations Completed 202 out of 500 iterations Completed 203 out of 500 iterations Completed 204 out of 500 iterations Completed 205 out of 500 iterations Completed 206 out of 500 iterations Completed 207 out of 500 iterations Completed 208 out of 500 iterations Completed 209 out of 500 iterations Completed 210 out of 500 iterations Completed 211 out of 500 iterations Completed 212 out of 500 iterations Completed 213 out of 500 iterations Completed 214 out of 500 iterations Completed 215 out of 500 iterations Completed 216 out of 500 iterations Completed 217 out of 500 iterations Completed 218 out of 500 iterations Completed 219 out of 500 iterations Completed 220 out of 500 iterations Completed 221 out of 500 iterations Completed 222 out of 500 iterations Completed 223 out of 500 iterations Completed 224 out of 500 iterations Completed 225 out of 500 iterations Completed 226 out of 500 iterations Completed 227 out of 500 iterations Completed 228 out of 500 iterations Completed 229 out of 500 iterations Completed 230 out of 500 iterations Completed 231 out of 500 iterations Completed 232 out of 500 iterations Completed 233 out of 500 iterations Completed 234 out of 500 iterations Completed 235 out of 500 iterations Completed 236 out of 500 iterations Completed 237 out of 500 iterations Completed 238 out of 500 iterations Completed 239 out of 500 iterations Completed 240 out of 500 iterations Completed 241 out of 500 iterations Completed 242 out of 500 iterations Completed 243 out of 500 iterations Completed 244 out of 500 iterations Completed 245 out of 500 iterations Completed 246 out of 500 iterations Completed 247 out of 500 iterations Completed 248 out of 500 iterations Completed 249 out of 500 iterations Completed 250 out of 500 iterations Completed 251 out of 500 iterations Completed 252 out of 500 iterations Completed 253 out of 500 iterations Completed 254 out of 500 iterations Completed 255 out of 500 iterations Completed 256 out of 500 iterations Completed 257 out of 500 iterations Completed 258 out of 500 iterations Completed 259 out of 500 iterations Completed 260 out of 500 iterations Completed 261 out of 500 iterations Completed 262 out of 500 iterations Completed 263 out of 500 iterations Completed 264 out of 500 iterations Completed 265 out of 500 iterations Completed 266 out of 500 iterations Completed 267 out of 500 iterations Completed 268 out of 500 iterations Completed 269 out of 500 iterations Completed 270 out of 500 iterations Completed 271 out of 500 iterations Completed 272 out of 500 iterations Completed 273 out of 500 iterations Completed 274 out of 500 iterations Completed 275 out of 500 iterations Completed 276 out of 500 iterations Completed 277 out of 500 iterations Completed 278 out of 500 iterations Completed 279 out of 500 iterations Completed 280 out of 500 iterations Completed 281 out of 500 iterations Completed 282 out of 500 iterations Completed 283 out of 500 iterations Completed 284 out of 500 iterations Completed 285 out of 500 iterations Completed 286 out of 500 iterations Completed 287 out of 500 iterations Completed 288 out of 500 iterations Completed 289 out of 500 iterations Completed 290 out of 500 iterations Completed 291 out of 500 iterations Completed 292 out of 500 iterations Completed 293 out of 500 iterations Completed 294 out of 500 iterations Completed 295 out of 500 iterations Completed 296 out of 500 iterations Completed 297 out of 500 iterations Completed 298 out of 500 iterations Completed 299 out of 500 iterations Completed 300 out of 500 iterations Completed 301 out of 500 iterations Completed 302 out of 500 iterations Completed 303 out of 500 iterations Completed 304 out of 500 iterations Completed 305 out of 500 iterations Completed 306 out of 500 iterations Completed 307 out of 500 iterations Completed 308 out of 500 iterations Completed 309 out of 500 iterations Completed 310 out of 500 iterations Completed 311 out of 500 iterations Completed 312 out of 500 iterations Completed 313 out of 500 iterations Completed 314 out of 500 iterations Completed 315 out of 500 iterations Completed 316 out of 500 iterations Completed 317 out of 500 iterations Completed 318 out of 500 iterations Completed 319 out of 500 iterations Completed 320 out of 500 iterations Completed 321 out of 500 iterations Completed 322 out of 500 iterations Completed 323 out of 500 iterations Completed 324 out of 500 iterations Completed 325 out of 500 iterations Completed 326 out of 500 iterations Completed 327 out of 500 iterations Completed 328 out of 500 iterations Completed 329 out of 500 iterations Completed 330 out of 500 iterations Completed 331 out of 500 iterations Completed 332 out of 500 iterations Completed 333 out of 500 iterations Completed 334 out of 500 iterations Completed 335 out of 500 iterations Completed 336 out of 500 iterations Completed 337 out of 500 iterations Completed 338 out of 500 iterations Completed 339 out of 500 iterations Completed 340 out of 500 iterations Completed 341 out of 500 iterations Completed 342 out of 500 iterations Completed 343 out of 500 iterations Completed 344 out of 500 iterations Completed 345 out of 500 iterations Completed 346 out of 500 iterations Completed 347 out of 500 iterations Completed 348 out of 500 iterations Completed 349 out of 500 iterations Completed 350 out of 500 iterations Completed 351 out of 500 iterations Completed 352 out of 500 iterations Completed 353 out of 500 iterations Completed 354 out of 500 iterations Completed 355 out of 500 iterations Completed 356 out of 500 iterations Completed 357 out of 500 iterations Completed 358 out of 500 iterations Completed 359 out of 500 iterations Completed 360 out of 500 iterations Completed 361 out of 500 iterations Completed 362 out of 500 iterations Completed 363 out of 500 iterations Completed 364 out of 500 iterations Completed 365 out of 500 iterations Completed 366 out of 500 iterations Completed 367 out of 500 iterations Completed 368 out of 500 iterations Completed 369 out of 500 iterations Completed 370 out of 500 iterations Completed 371 out of 500 iterations Completed 372 out of 500 iterations Completed 373 out of 500 iterations Completed 374 out of 500 iterations Completed 375 out of 500 iterations Completed 376 out of 500 iterations Completed 377 out of 500 iterations Completed 378 out of 500 iterations Completed 379 out of 500 iterations Completed 380 out of 500 iterations Completed 381 out of 500 iterations Completed 382 out of 500 iterations Completed 383 out of 500 iterations Completed 384 out of 500 iterations Completed 385 out of 500 iterations Completed 386 out of 500 iterations Completed 387 out of 500 iterations Completed 388 out of 500 iterations Completed 389 out of 500 iterations Completed 390 out of 500 iterations Completed 391 out of 500 iterations Completed 392 out of 500 iterations Completed 393 out of 500 iterations Completed 394 out of 500 iterations Completed 395 out of 500 iterations Completed 396 out of 500 iterations Completed 397 out of 500 iterations Completed 398 out of 500 iterations Completed 399 out of 500 iterations Completed 400 out of 500 iterations Completed 401 out of 500 iterations Completed 402 out of 500 iterations Completed 403 out of 500 iterations Completed 404 out of 500 iterations Completed 405 out of 500 iterations Completed 406 out of 500 iterations Completed 407 out of 500 iterations Completed 408 out of 500 iterations Completed 409 out of 500 iterations Completed 410 out of 500 iterations Completed 411 out of 500 iterations Completed 412 out of 500 iterations Completed 413 out of 500 iterations Completed 414 out of 500 iterations Completed 415 out of 500 iterations Completed 416 out of 500 iterations Completed 417 out of 500 iterations Completed 418 out of 500 iterations Completed 419 out of 500 iterations Completed 420 out of 500 iterations Completed 421 out of 500 iterations Completed 422 out of 500 iterations Completed 423 out of 500 iterations Completed 424 out of 500 iterations Completed 425 out of 500 iterations Completed 426 out of 500 iterations Completed 427 out of 500 iterations Completed 428 out of 500 iterations Completed 429 out of 500 iterations Completed 430 out of 500 iterations Completed 431 out of 500 iterations Completed 432 out of 500 iterations Completed 433 out of 500 iterations Completed 434 out of 500 iterations Completed 435 out of 500 iterations Completed 436 out of 500 iterations Completed 437 out of 500 iterations Completed 438 out of 500 iterations Completed 439 out of 500 iterations Completed 440 out of 500 iterations Completed 441 out of 500 iterations Completed 442 out of 500 iterations Completed 443 out of 500 iterations Completed 444 out of 500 iterations Completed 445 out of 500 iterations Completed 446 out of 500 iterations Completed 447 out of 500 iterations Completed 448 out of 500 iterations Completed 449 out of 500 iterations Completed 450 out of 500 iterations Completed 451 out of 500 iterations Completed 452 out of 500 iterations Completed 453 out of 500 iterations Completed 454 out of 500 iterations Completed 455 out of 500 iterations Completed 456 out of 500 iterations Completed 457 out of 500 iterations Completed 458 out of 500 iterations Completed 459 out of 500 iterations Completed 460 out of 500 iterations Completed 461 out of 500 iterations Completed 462 out of 500 iterations Completed 463 out of 500 iterations Completed 464 out of 500 iterations Completed 465 out of 500 iterations Completed 466 out of 500 iterations Completed 467 out of 500 iterations Completed 468 out of 500 iterations Completed 469 out of 500 iterations Completed 470 out of 500 iterations Completed 471 out of 500 iterations Completed 472 out of 500 iterations Completed 473 out of 500 iterations Completed 474 out of 500 iterations Completed 475 out of 500 iterations Completed 476 out of 500 iterations Completed 477 out of 500 iterations Completed 478 out of 500 iterations Completed 479 out of 500 iterations Completed 480 out of 500 iterations Completed 481 out of 500 iterations Completed 482 out of 500 iterations Completed 483 out of 500 iterations Completed 484 out of 500 iterations Completed 485 out of 500 iterations Completed 486 out of 500 iterations Completed 487 out of 500 iterations Completed 488 out of 500 iterations Completed 489 out of 500 iterations Completed 490 out of 500 iterations Completed 491 out of 500 iterations Completed 492 out of 500 iterations Completed 493 out of 500 iterations Completed 494 out of 500 iterations Completed 495 out of 500 iterations Completed 496 out of 500 iterations Completed 497 out of 500 iterations Completed 498 out of 500 iterations Completed 499 out of 500 iterations Completed 500 out of 500 iterations Best C for Polynomial Kernel: 0.1 Validation error for Polynomial Kernel: 0.048
Radial Basis Function Kernel¶
In [ ]:
gamma_low = -5 ; gamma_high = 5
C_low = -5 ; C_high = 5
gamma_values = np.logspace(gamma_low, gamma_high, gamma_high-gamma_low+1, base=10)
C_values = np.logspace(gamma_low, gamma_high, C_high-C_low+1, base=10)
mse_values = np.zeros((len(gamma_values), len(C_values)))
total_iterations = len(gamma_values) * len(C_values)
iteration_count = 0
#Computing the kernel matrices
K_rbf = rbf_kernel(X_train.T, X_train.T)
for i, gamma in enumerate(gamma_values):
for j, c in enumerate(C_values):
clf_rbf = svm.SVC(kernel='precomputed', C=c)
clf_rbf.fit(K_rbf, y_train)
# Validating the model
K_val_rbf = rbf_kernel(X_Val.T,X_train.T, gamma=gamma)
# Predict on validation set
Yval_Pred_rbf = clf_rbf.predict(K_val_rbf)
# mse_val_value = mean_squared_error(y_Val, Yval_Pred_rbf)
mse_val_value = np.mean((y_Val - Yval_Pred_rbf)**2)
mse_values[i, j] = mse_val_value
# % Complete status update
iteration_count += 1
print("Completed ", iteration_count, " out of ", total_iterations, " iterations")
best_gamma_index, best_C_index = np.unravel_index(np.argmin(mse_values), mse_values.shape)
best_gamma = gamma_values[best_gamma_index]
best_C = C_values[best_C_index]
best_val_MSE = mse_values[best_gamma_index, best_C_index]
print("Best gamma: ", best_gamma)
print("Best C: ", best_C)
print(f"Best validation MSE: {best_val_MSE:.7f}")
# Training the model with the best hyperparameters on the test set
clf_rbf_best = svm.SVC(kernel='precomputed', C=best_C)
clf_rbf_best.fit(K_rbf, y_train)
K_test_rbf = rbf_kernel(X_test.T, X_train.T,gamma=best_gamma)
y_pred_rbf = clf_rbf_best.predict(K_test_rbf)
test_MMSE_rbf = np.mean((y_test - y_pred_rbf)**2)
# Generate a grid of points
x_min_plot = X_test[0, :].min() - 1
x_max_plot = X_test[0, :].max() + 1
y_min_plot = X_test[1, :].min() - 1
y_max_plot = X_test[1, :].max() + 1
x_meshgrid, y_meshgrid = np.meshgrid(np.linspace(x_min_plot, x_max_plot, 100), np.linspace(y_min_plot, y_max_plot, 100))
# Combine x_meshgrid and y_meshgri into a single matrix
grid_points = np.c_[x_meshgrid.ravel(), y_meshgrid.ravel()]
# Compute kernel values between grid points and training data for each kernel
K_test_rbf = rbf_kernel(grid_points, X_train.T, gamma=best_gamma)
# Predict on grid points for each kernel
Z_rbf = clf_rbf_best.decision_function(K_test_rbf)
# Reshape the decision function values
Z_rbf = Z_rbf.reshape(x_meshgrid.shape)
# Plot decision boundary for each kernel
plt.figure(figsize=(5, 5))
plt.contour(x_meshgrid, y_meshgrid, Z_rbf, levels=[0], linewidths=2, colors='green')
plt.scatter(X_test[0, y_test == 1], X_test[1, y_test == 1], c='blue', marker='*', label='Class 1')
plt.scatter(X_test[0, y_test == -1], X_test[1, y_test == -1], c='red', marker='*', label='Class -1')
plt.xlabel('x[n-1]')
plt.ylabel('x[n]')
plt.title('Radial Basis Function')
plt.show()
Completed 1 out of 121 iterations Completed 2 out of 121 iterations Completed 3 out of 121 iterations Completed 4 out of 121 iterations Completed 5 out of 121 iterations Completed 6 out of 121 iterations Completed 7 out of 121 iterations Completed 8 out of 121 iterations Completed 9 out of 121 iterations Completed 10 out of 121 iterations Completed 11 out of 121 iterations Completed 12 out of 121 iterations Completed 13 out of 121 iterations Completed 14 out of 121 iterations Completed 15 out of 121 iterations Completed 16 out of 121 iterations Completed 17 out of 121 iterations Completed 18 out of 121 iterations Completed 19 out of 121 iterations Completed 20 out of 121 iterations Completed 21 out of 121 iterations Completed 22 out of 121 iterations Completed 23 out of 121 iterations Completed 24 out of 121 iterations Completed 25 out of 121 iterations Completed 26 out of 121 iterations Completed 27 out of 121 iterations Completed 28 out of 121 iterations Completed 29 out of 121 iterations Completed 30 out of 121 iterations Completed 31 out of 121 iterations Completed 32 out of 121 iterations Completed 33 out of 121 iterations Completed 34 out of 121 iterations Completed 35 out of 121 iterations Completed 36 out of 121 iterations Completed 37 out of 121 iterations Completed 38 out of 121 iterations Completed 39 out of 121 iterations Completed 40 out of 121 iterations Completed 41 out of 121 iterations Completed 42 out of 121 iterations Completed 43 out of 121 iterations Completed 44 out of 121 iterations Completed 45 out of 121 iterations Completed 46 out of 121 iterations Completed 47 out of 121 iterations Completed 48 out of 121 iterations Completed 49 out of 121 iterations Completed 50 out of 121 iterations Completed 51 out of 121 iterations Completed 52 out of 121 iterations Completed 53 out of 121 iterations Completed 54 out of 121 iterations Completed 55 out of 121 iterations Completed 56 out of 121 iterations Completed 57 out of 121 iterations Completed 58 out of 121 iterations Completed 59 out of 121 iterations Completed 60 out of 121 iterations Completed 61 out of 121 iterations Completed 62 out of 121 iterations Completed 63 out of 121 iterations Completed 64 out of 121 iterations Completed 65 out of 121 iterations Completed 66 out of 121 iterations Completed 67 out of 121 iterations Completed 68 out of 121 iterations Completed 69 out of 121 iterations Completed 70 out of 121 iterations Completed 71 out of 121 iterations Completed 72 out of 121 iterations Completed 73 out of 121 iterations Completed 74 out of 121 iterations Completed 75 out of 121 iterations Completed 76 out of 121 iterations Completed 77 out of 121 iterations Completed 78 out of 121 iterations Completed 79 out of 121 iterations Completed 80 out of 121 iterations Completed 81 out of 121 iterations Completed 82 out of 121 iterations Completed 83 out of 121 iterations Completed 84 out of 121 iterations Completed 85 out of 121 iterations Completed 86 out of 121 iterations Completed 87 out of 121 iterations Completed 88 out of 121 iterations Completed 89 out of 121 iterations Completed 90 out of 121 iterations Completed 91 out of 121 iterations Completed 92 out of 121 iterations Completed 93 out of 121 iterations Completed 94 out of 121 iterations Completed 95 out of 121 iterations Completed 96 out of 121 iterations Completed 97 out of 121 iterations Completed 98 out of 121 iterations Completed 99 out of 121 iterations Completed 100 out of 121 iterations Completed 101 out of 121 iterations Completed 102 out of 121 iterations Completed 103 out of 121 iterations Completed 104 out of 121 iterations Completed 105 out of 121 iterations Completed 106 out of 121 iterations Completed 107 out of 121 iterations Completed 108 out of 121 iterations Completed 109 out of 121 iterations Completed 110 out of 121 iterations Completed 111 out of 121 iterations Completed 112 out of 121 iterations Completed 113 out of 121 iterations Completed 114 out of 121 iterations Completed 115 out of 121 iterations Completed 116 out of 121 iterations Completed 117 out of 121 iterations Completed 118 out of 121 iterations Completed 119 out of 121 iterations Completed 120 out of 121 iterations Completed 121 out of 121 iterations Best gamma: 1.0 Best C: 0.1 Best validation MSE: 0.0000000
In [ ]:
gamma_low = 0.5 ; gamma_high = 1.5
C_low = 0.01 ; C_high = 1
gamma_values = np.linspace(gamma_low, gamma_high, num=500)
C_values = np.linspace(gamma_low, gamma_high, num=500)
mse_values = np.zeros((len(gamma_values), len(C_values)))
total_iterations = len(gamma_values) * len(C_values)
iteration_count = 0
#Computing the kernel matrices
K_rbf = rbf_kernel(X_train.T, X_train.T)
for i, gamma in enumerate(gamma_values):
for j, c in enumerate(C_values):
clf_rbf = svm.SVC(kernel='precomputed', C=c)
clf_rbf.fit(K_rbf, y_train)
# Validating the model
K_val_rbf = rbf_kernel(X_Val.T,X_train.T, gamma=gamma)
# Predict on validation set
Yval_Pred_rbf = clf_rbf.predict(K_val_rbf)
# mse_val_value = mean_squared_error(y_Val, Yval_Pred_rbf)
mse_val_value = np.mean((y_Val - Yval_Pred_rbf)**2)
mse_values[i, j] = mse_val_value
# % Complete status update
iteration_count += 1
print("Completed ", iteration_count, " out of ", total_iterations, " iterations")
best_gamma_index, best_C_index = np.unravel_index(np.argmin(mse_values), mse_values.shape)
best_gamma = gamma_values[best_gamma_index]
best_C = C_values[best_C_index]
best_val_MSE = mse_values[best_gamma_index, best_C_index]
print("Best gamma: ", best_gamma)
print("Best C: ", best_C)
print(f"Best validation MSE: {best_val_MSE:.7f}")
# Training the model with the best hyperparameters on the test set
clf_rbf_best = svm.SVC(kernel='precomputed', C=best_C)
clf_rbf_best.fit(K_rbf, y_train)
K_test_rbf = rbf_kernel(X_test.T, X_train.T,gamma=best_gamma)
y_pred_rbf = clf_rbf_best.predict(K_test_rbf)
test_MMSE_rbf = np.mean((y_test - y_pred_rbf)**2)
# Generate a grid of points
x_min_plot = X_test[0, :].min() - 1
x_max_plot = X_test[0, :].max() + 1
y_min_plot = X_test[1, :].min() - 1
y_max_plot = X_test[1, :].max() + 1
x_meshgrid, y_meshgrid = np.meshgrid(np.linspace(x_min_plot, x_max_plot, 100), np.linspace(y_min_plot, y_max_plot, 100))
# Combine x_meshgrid and y_meshgri into a single matrix
grid_points = np.c_[x_meshgrid.ravel(), y_meshgrid.ravel()]
# Compute kernel values between grid points and training data for each kernel
K_test_rbf = rbf_kernel(grid_points, X_train.T, gamma=best_gamma)
# Predict on grid points for each kernel
Z_rbf = clf_rbf_best.decision_function(K_test_rbf)
# Reshape the decision function values
Z_rbf = Z_rbf.reshape(x_meshgrid.shape)
# Plot decision boundary for each kernel
plt.figure(figsize=(5, 5))
plt.contour(x_meshgrid, y_meshgrid, Z_rbf, levels=[0], linewidths=2, colors='green')
plt.scatter(X_test[0, y_test == 1], X_test[1, y_test == 1], c='blue', marker='*', label='Class 1')
plt.scatter(X_test[0, y_test == -1], X_test[1, y_test == -1], c='red', marker='*', label='Class -1')
plt.xlabel('x[n-1]')
plt.ylabel('x[n]')
plt.title('Radial Basis Function')
plt.show()